{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## by： AI蜗牛车\n",
    "## 公众号： AI蜗牛车\n",
    "## wx：Che_Hongshu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo1-TensorDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = pd.read_csv('PRSA_data_2010.1.1-2014.12.31.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>pm2.5</th>\n",
       "      <th>DEWP</th>\n",
       "      <th>TEMP</th>\n",
       "      <th>PRES</th>\n",
       "      <th>cbwd</th>\n",
       "      <th>Iws</th>\n",
       "      <th>Is</th>\n",
       "      <th>Ir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-21</td>\n",
       "      <td>-11.0</td>\n",
       "      <td>1021.0</td>\n",
       "      <td>NW</td>\n",
       "      <td>1.79</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-21</td>\n",
       "      <td>-12.0</td>\n",
       "      <td>1020.0</td>\n",
       "      <td>NW</td>\n",
       "      <td>4.92</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-21</td>\n",
       "      <td>-11.0</td>\n",
       "      <td>1019.0</td>\n",
       "      <td>NW</td>\n",
       "      <td>6.71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-21</td>\n",
       "      <td>-14.0</td>\n",
       "      <td>1019.0</td>\n",
       "      <td>NW</td>\n",
       "      <td>9.84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-20</td>\n",
       "      <td>-12.0</td>\n",
       "      <td>1018.0</td>\n",
       "      <td>NW</td>\n",
       "      <td>12.97</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   No  year  month  day  hour  pm2.5  DEWP  TEMP    PRES cbwd    Iws  Is  Ir\n",
       "0   1  2010      1    1     0    NaN   -21 -11.0  1021.0   NW   1.79   0   0\n",
       "1   2  2010      1    1     1    NaN   -21 -12.0  1020.0   NW   4.92   0   0\n",
       "2   3  2010      1    1     2    NaN   -21 -11.0  1019.0   NW   6.71   0   0\n",
       "3   4  2010      1    1     3    NaN   -21 -14.0  1019.0   NW   9.84   0   0\n",
       "4   5  2010      1    1     4    NaN   -20 -12.0  1018.0   NW  12.97   0   0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_col =  ['No', 'year', 'month', 'day', 'hour', 'DEWP', 'TEMP', 'PRES', 'Iws', 'Is', 'Ir']\n",
    "data_df_x = data_df.loc[:127,feature_col]\n",
    "label_col = ['pm2.5']\n",
    "data_df_y = data_df.loc[:127, label_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_numpy_x = data_df_x.values\n",
    "data_numpy_y = data_df_y.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.from_numpy(data_numpy_x)\n",
    "Y = torch.from_numpy(data_numpy_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = TensorDataset(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader = DataLoader(dataset=dataset,\n",
    "          batch_size=64,\n",
    "          shuffle=True,\n",
    "          num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "<class 'torch.Tensor'> <class 'torch.Tensor'>\n",
      "torch.Size([64, 11]) torch.Size([64, 1])\n",
      "1\n",
      "<class 'torch.Tensor'> <class 'torch.Tensor'>\n",
      "torch.Size([64, 11]) torch.Size([64, 1])\n"
     ]
    }
   ],
   "source": [
    "for i, data in enumerate(data_loader):\n",
    "    print(i)\n",
    "    x, y = data\n",
    "    print(type(x), type(y))\n",
    "    print(x.data.size(), y.data.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo2-自定义Dataset(结构化数据)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset,DataLoader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "<class 'torch.Tensor'> <class 'torch.Tensor'>\n",
      "torch.Size([64, 11]) torch.Size([64, 1])\n",
      "1\n",
      "<class 'torch.Tensor'> <class 'torch.Tensor'>\n",
      "torch.Size([64, 11]) torch.Size([64, 1])\n"
     ]
    }
   ],
   "source": [
    "class MyDataset(Dataset):\n",
    "    \n",
    "    def __init__(self):\n",
    "        data_df = pd.read_csv('PRSA_data_2010.1.1-2014.12.31.csv')\n",
    "        feature_col =  ['No', 'year', 'month', 'day', 'hour', 'DEWP', 'TEMP', 'PRES', 'Iws', 'Is', 'Ir']\n",
    "        data_df_x = data_df.loc[:127,feature_col]\n",
    "        label_col = ['pm2.5']\n",
    "        data_df_y = data_df.loc[:127, label_col]\n",
    "        data_numpy_x = data_df_x.values\n",
    "        data_numpy_y = data_df_y.values\n",
    "        self.X = torch.from_numpy(data_numpy_x)\n",
    "        self.Y = torch.from_numpy(data_numpy_y)\n",
    "        self.len = data_numpy_x.shape[0]\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        return self.X[index], self.Y[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "# 实例化这个类，和上面的 直接咱们用官方的TensorDataset是一样的\n",
    "dataset2= MyDataset()\n",
    "\n",
    "data_loader2 = DataLoader(dataset=dataset2,\n",
    "                          batch_size=64,\n",
    "                          shuffle=True)\n",
    "\n",
    "for i, data in enumerate(data_loader2):\n",
    "    print(i)\n",
    "    x, y = data\n",
    "    print(type(x), type(y))\n",
    "    print(x.data.size(), y.data.size())   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo3-自定义Dataset(图片数据)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader,Dataset\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import torch\n",
    "from torchvision import transforms\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyData_Animal(Dataset): \n",
    "    def __init__(self, root_dir, transform=None): \n",
    "        self.root_dir = root_dir  \n",
    "        self.transform = transform \n",
    "        self.images_path_list = os.listdir(self.root_dir) \n",
    "    \n",
    "    def __len__(self): \n",
    "        return len(self.images_path_list)\n",
    "    \n",
    "    def __getitem__(self,index): \n",
    "        image_index = self.images_path_list[index]\n",
    "        img_path = os.path.join(self.root_dir, image_index)\n",
    "        img = Image.open(img_path)\n",
    "        img = img.resize((64, 64))\n",
    "        label = img_path.split('\\\\')[-1].split('.')[0]\n",
    "        \n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "        \n",
    "        return img, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "<class 'torch.Tensor'> <class 'tuple'>\n",
      "torch.Size([4, 3, 64, 64]) ('dog', 'cat', 'cat', 'dog')\n",
      "1\n",
      "<class 'torch.Tensor'> <class 'tuple'>\n",
      "torch.Size([4, 3, 64, 64]) ('cat', 'dog', 'cat', 'dog')\n",
      "2\n",
      "<class 'torch.Tensor'> <class 'tuple'>\n",
      "torch.Size([4, 3, 64, 64]) ('dog', 'dog', 'dog', 'cat')\n",
      "3\n",
      "<class 'torch.Tensor'> <class 'tuple'>\n",
      "torch.Size([4, 3, 64, 64]) ('cat', 'cat', 'dog', 'dog')\n",
      "4\n",
      "<class 'torch.Tensor'> <class 'tuple'>\n",
      "torch.Size([4, 3, 64, 64]) ('dog', 'dog', 'dog', 'dog')\n",
      "5\n",
      "<class 'torch.Tensor'> <class 'tuple'>\n",
      "torch.Size([2, 3, 64, 64]) ('cat', 'dog')\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose([\n",
    "#     transforms.Resize(64), \n",
    "    transforms.ToTensor(),\n",
    "    #transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) # 归一化\n",
    "])\n",
    "\n",
    "mydataset_animal = MyData_Animal('./train', transform)\n",
    "\n",
    "data_loader3 = DataLoader(dataset=mydataset_animal,\n",
    "                          batch_size=4,\n",
    "                          shuffle=True)\n",
    "\n",
    "\n",
    "\n",
    "for i, (img, label) in enumerate(data_loader3):\n",
    "    print(i)\n",
    "    print(type(img), type(label))\n",
    "    print(img.data.size(), label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mydataset_animal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pych",
   "language": "python",
   "name": "pych"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
